

from langchain_text_splitters.base import (
    Language,
    TextSplitter,
    Tokenizer,
    TokenTextSplitter,
    split_text_on_tokens,
)
from langchain_text_splitters.character import (
    CharacterTextSplitter,
    RecursiveCharacterTextSplitter,
)
from langchain_text_splitters.html import (
    ElementType,
    HTMLHeaderTextSplitter,
    HTMLSectionSplitter,
    HTMLSemanticPreservingSplitter,
)
from langchain_text_splitters.json import RecursiveJsonSplitter
from langchain_text_splitters.jsx import JSFrameworkTextSplitter
from langchain_text_splitters.konlpy import KonlpyTextSplitter
from langchain_text_splitters.latex import LatexTextSplitter
from langchain_text_splitters.markdown import (
    ExperimentalMarkdownSyntaxTextSplitter,
    HeaderType,
    LineType,
    MarkdownHeaderTextSplitter,
    MarkdownTextSplitter,
)
from langchain_text_splitters.nltk import NLTKTextSplitter
from langchain_text_splitters.python import PythonCodeTextSplitter
from langchain_text_splitters.sentence_transformers import (
    SentenceTransformersTokenTextSplitter,
)
from langchain_text_splitters.spacy import SpacyTextSplitter

from langchain.text_splitter import SentenceTransformersTokenTextSplitter
from sentence_transformers import SentenceTransformer

# 初始化嵌入模型和分割器
model = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2")
text_splitter = SentenceTransformersTokenTextSplitter(
    model_name="paraphrase-multilingual-MiniLM-L12-v2",
    chunk_size=100,
    chunk_overlap=20
)

# 分割中文文本
text = "10月15日起，2026年度国家公务员考试开始报名，今年国考计划招录3.81万人，报考年龄限制放宽至本科38周岁以下、应届硕博43周岁以下，笔试定于11月30日举行。"
chunks = text_splitter.split_text(text)

print(chunks)

